{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>cmn</th>\n",
       "      <th>我們試試看！</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>cmn</td>\n",
       "      <td>我该去睡觉了。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>cmn</td>\n",
       "      <td>你在干什麼啊？</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>cmn</td>\n",
       "      <td>這是什麼啊？</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>cmn</td>\n",
       "      <td>今天是６月１８号，也是Muiriel的生日！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>cmn</td>\n",
       "      <td>生日快乐，Muiriel！</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   1  cmn                  我們試試看！\n",
       "0  2  cmn                 我该去睡觉了。\n",
       "1  3  cmn                 你在干什麼啊？\n",
       "2  4  cmn                  這是什麼啊？\n",
       "3  5  cmn  今天是６月１８号，也是Muiriel的生日！\n",
       "4  6  cmn           生日快乐，Muiriel！"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lang = pd.read_csv('sentences.csv',sep='\\t')\n",
    "lang = lang.dropna()\n",
    "lang.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 cmn\n",
      "1 deu\n",
      "2 rus\n",
      "3 fra\n",
      "4 eng\n",
      "5 jpn\n",
      "6 spa\n",
      "7 ita\n",
      "8 kor\n",
      "9 vie\n",
      "10 nld\n",
      "11 epo\n",
      "12 tur\n",
      "13 por\n",
      "14 heb\n",
      "15 hun\n",
      "16 ell\n",
      "17 ind\n",
      "18 ara\n",
      "19 arz\n",
      "20 fin\n",
      "21 bul\n",
      "22 yue\n",
      "23 swe\n",
      "24 ukr\n",
      "25 bel\n",
      "27 ces\n",
      "29 nno\n",
      "30 wuu\n",
      "31 nob\n",
      "32 zsm\n",
      "33 est\n",
      "34 kat\n",
      "35 pol\n",
      "36 lat\n",
      "37 urd\n",
      "38 sqi\n",
      "39 isl\n",
      "41 afr\n",
      "42 ron\n",
      "45 bre\n",
      "46 tat\n",
      "47 yid\n",
      "48 uig\n",
      "49 uzb\n",
      "50 srp\n",
      "52 dan\n",
      "53 pes\n",
      "54 slk\n",
      "55 eus\n",
      "58 tgl\n",
      "59 lvs\n",
      "60 kaz\n",
      "61 hye\n",
      "62 hin\n",
      "63 lit\n",
      "64 ben\n",
      "65 cat\n",
      "67 hrv\n",
      "68 tha\n",
      "69 orv\n",
      "71 mon\n",
      "72 lzh\n",
      "74 gle\n",
      "75 mkd\n",
      "76 slv\n",
      "78 glg\n",
      "79 vol\n",
      "81 jbo\n",
      "82 toki\n",
      "83 ina\n",
      "84 nds\n",
      "85 mal\n",
      "86 tlh\n",
      "90 ido\n",
      "91 gla\n",
      "96 oci\n",
      "97 ile\n",
      "98 xal\n",
      "102 khm\n",
      "105 aze\n",
      "108 chv\n",
      "109 hsb\n",
      "110 dsb\n",
      "113 cym\n",
      "116 kur\n",
      "118 kab\n",
      "119 ber\n",
      "120 tpw\n",
      "123 pms\n",
      "125 grn\n",
      "129 grc\n",
      "133 cor\n",
      "135 avk\n",
      "136 mar\n",
      "141 prg\n",
      "148 cbk\n",
      "153 arq\n",
      "157 rom\n",
      "166 mhr\n",
      "196 sah\n",
      "222 kha\n",
      "223 ceb\n",
      "228 lfn\n",
      "236 kzj\n",
      "237 pam\n",
      "246 zza\n",
      "254 dtp\n",
      "260 ilo\n",
      "264 hoc\n",
      "268 war\n",
      "273 csb\n",
      "289 nst\n"
     ]
    }
   ],
   "source": [
    "X, Y = [], []\n",
    "for no, ln in enumerate(lang.cmn.unique()):\n",
    "    langs = lang.loc[lang.cmn == ln]\n",
    "    if langs.shape[0] < 500:\n",
    "        continue\n",
    "    print(no, ln)\n",
    "    langs = langs.iloc[:500,-1].tolist()\n",
    "    X.extend(langs)\n",
    "    Y.extend([ln] * len(langs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "def clean_text(string):\n",
    "    string = re.sub(u'[0-9!@#$%^&*()_\\-+{}|\\~`\\'\";:?/.>,<]', ' ', string.lower(), flags=re.UNICODE)\n",
    "    return re.sub(r'[ ]+', ' ', string.lower()).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X =[clean_text(s) for s in X]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56500, 688217)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_chars = CountVectorizer(ngram_range=(3, 5), analyzer='char_wb', max_features=700000).fit(X)\n",
    "delattr(bow_chars, 'stop_words_')\n",
    "target = LabelEncoder().fit_transform(Y)\n",
    "features = bow_chars.transform(X)\n",
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(features, target, test_size = 0.2)\n",
    "del features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_sparse_matrix_to_sparse_tensor(X, limit = 5):\n",
    "    coo = X.tocoo()\n",
    "    indices = np.mat([coo.row, coo.col]).transpose()\n",
    "    coo.data[coo.data > limit] = limit\n",
    "    return tf.SparseTensorValue(indices, coo.col, coo.shape), tf.SparseTensorValue(indices, coo.data, coo.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['afr', 'ara', 'arq', 'arz', 'avk', 'aze', 'bel', 'ben', 'ber',\n",
       "       'bre', 'bul', 'cat', 'cbk', 'ceb', 'ces', 'chv', 'cmn', 'cor',\n",
       "       'csb', 'cym', 'dan', 'deu', 'dsb', 'dtp', 'ell', 'eng', 'epo',\n",
       "       'est', 'eus', 'fin', 'fra', 'gla', 'gle', 'glg', 'grc', 'grn',\n",
       "       'heb', 'hin', 'hoc', 'hrv', 'hsb', 'hun', 'hye', 'ido', 'ile',\n",
       "       'ilo', 'ina', 'ind', 'isl', 'ita', 'jbo', 'jpn', 'kab', 'kat',\n",
       "       'kaz', 'kha', 'khm', 'kor', 'kur', 'kzj', 'lat', 'lfn', 'lit',\n",
       "       'lvs', 'lzh', 'mal', 'mar', 'mhr', 'mkd', 'mon', 'nds', 'nld',\n",
       "       'nno', 'nob', 'nst', 'oci', 'orv', 'pam', 'pes', 'pms', 'pol',\n",
       "       'por', 'prg', 'rom', 'ron', 'rus', 'sah', 'slk', 'slv', 'spa',\n",
       "       'sqi', 'srp', 'swe', 'tat', 'tgl', 'tha', 'tlh', 'toki', 'tpw',\n",
       "       'tur', 'uig', 'ukr', 'urd', 'uzb', 'vie', 'vol', 'war', 'wuu',\n",
       "       'xal', 'yid', 'yue', 'zsm', 'zza'], dtype='<U4')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = np.unique(Y,return_counts=True)[0]\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, learning_rate):\n",
    "        self.X = tf.sparse_placeholder(tf.int32)\n",
    "        self.W = tf.sparse_placeholder(tf.int32)\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        embeddings = tf.Variable(tf.truncated_normal([train_X.shape[1], 64]))\n",
    "        embed = tf.nn.embedding_lookup_sparse(embeddings, self.X, self.W, combiner='mean')\n",
    "        self.logits = tf.layers.dense(embed, len(labels))\n",
    "        self.cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "            logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1,output_type=tf.int32), self.Y)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "model = Model(1e-4)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.56it/s, accuracy=0.0625, cost=4.65]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 398.26it/s, accuracy=0.0278, cost=4.73]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.71it/s, accuracy=0.0625, cost=4.68]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.243767976760864\n",
      "epoch: 0, training loss: 4.724888, training acc: 0.013584, valid loss: 4.713782, valid acc: 0.021131\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.0625, cost=4.6] \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 405.55it/s, accuracy=0.0278, cost=4.7] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.58it/s, accuracy=0.0938, cost=4.64]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.123547315597534\n",
      "epoch: 1, training loss: 4.684196, training acc: 0.036858, valid loss: 4.676456, valid acc: 0.050777\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.62it/s, accuracy=0.125, cost=4.55] \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 403.38it/s, accuracy=0.0833, cost=4.66]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.63it/s, accuracy=0.125, cost=4.59]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.13985466957092\n",
      "epoch: 2, training loss: 4.639668, training acc: 0.086416, valid loss: 4.635060, valid acc: 0.106844\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.62it/s, accuracy=0.188, cost=4.49] \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 405.60it/s, accuracy=0.167, cost=4.61] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.203, cost=4.53]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.13983368873596\n",
      "epoch: 3, training loss: 4.589314, training acc: 0.156836, valid loss: 4.587393, valid acc: 0.173510\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.438, cost=4.42] \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.57it/s, accuracy=0.167, cost=4.56]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.281, cost=4.47]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.12689709663391\n",
      "epoch: 4, training loss: 4.530825, training acc: 0.234159, valid loss: 4.531418, valid acc: 0.237493\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.61it/s, accuracy=0.625, cost=4.34]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 409.29it/s, accuracy=0.222, cost=4.5] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.68it/s, accuracy=0.375, cost=4.39]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.145896673202515\n",
      "epoch: 5, training loss: 4.462077, training acc: 0.308208, valid loss: 4.465344, valid acc: 0.294798\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=0.625, cost=4.24]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.52it/s, accuracy=0.278, cost=4.43]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.453, cost=4.3] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.110512018203735\n",
      "epoch: 6, training loss: 4.381257, training acc: 0.372854, valid loss: 4.387712, valid acc: 0.342901\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.688, cost=4.13]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.10it/s, accuracy=0.306, cost=4.35]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.68it/s, accuracy=0.516, cost=4.19]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.117857456207275\n",
      "epoch: 7, training loss: 4.286958, training acc: 0.427279, valid loss: 4.297482, valid acc: 0.388279\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=0.688, cost=4]   \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.52it/s, accuracy=0.333, cost=4.26]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.578, cost=4.07]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11299705505371\n",
      "epoch: 8, training loss: 4.178279, training acc: 0.471571, valid loss: 4.194131, valid acc: 0.423038\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.688, cost=3.85]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.36it/s, accuracy=0.361, cost=4.15]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.75it/s, accuracy=0.625, cost=3.93]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.12467956542969\n",
      "epoch: 9, training loss: 4.054937, training acc: 0.509956, valid loss: 4.077751, valid acc: 0.455320\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.75, cost=3.69] \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 405.53it/s, accuracy=0.417, cost=4.03]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.59it/s, accuracy=0.672, cost=3.77]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.124980211257935\n",
      "epoch: 10, training loss: 3.917367, training acc: 0.543496, valid loss: 3.949126, valid acc: 0.481740\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.812, cost=3.52]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 404.65it/s, accuracy=0.444, cost=3.9] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.63it/s, accuracy=0.703, cost=3.6] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.12359809875488\n",
      "epoch: 11, training loss: 3.766787, training acc: 0.573739, valid loss: 3.809739, valid acc: 0.508446\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.812, cost=3.34]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 400.58it/s, accuracy=0.472, cost=3.76]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.48it/s, accuracy=0.719, cost=3.42]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.13361191749573\n",
      "epoch: 12, training loss: 3.605150, training acc: 0.602876, valid loss: 3.661636, valid acc: 0.535683\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.62it/s, accuracy=0.812, cost=3.15]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.04it/s, accuracy=0.472, cost=3.62]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.65it/s, accuracy=0.75, cost=3.24] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.136579513549805\n",
      "epoch: 13, training loss: 3.434938, training acc: 0.629757, valid loss: 3.507177, valid acc: 0.559843\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.62it/s, accuracy=0.812, cost=2.95]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.85it/s, accuracy=0.556, cost=3.47]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.76it/s, accuracy=0.766, cost=3.05]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.140838384628296\n",
      "epoch: 14, training loss: 3.258861, training acc: 0.655509, valid loss: 3.348779, valid acc: 0.584120\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.875, cost=2.76]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.02it/s, accuracy=0.583, cost=3.31]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.64it/s, accuracy=0.781, cost=2.86]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.12215447425842\n",
      "epoch: 15, training loss: 3.079584, training acc: 0.680907, valid loss: 3.188723, valid acc: 0.608260\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=0.938, cost=2.58]\n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 404.64it/s, accuracy=0.583, cost=3.15]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.63it/s, accuracy=0.812, cost=2.67]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.117107629776\n",
      "epoch: 16, training loss: 2.899566, training acc: 0.705619, valid loss: 3.029056, valid acc: 0.629056\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=0.938, cost=2.4] \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 405.76it/s, accuracy=0.583, cost=3]   \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.844, cost=2.48]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.13151812553406\n",
      "epoch: 17, training loss: 2.720992, training acc: 0.729757, valid loss: 2.871556, valid acc: 0.651976\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=2.23]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.87it/s, accuracy=0.611, cost=2.84]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.74it/s, accuracy=0.844, cost=2.29]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11476802825928\n",
      "epoch: 18, training loss: 2.545758, training acc: 0.751858, valid loss: 2.717727, valid acc: 0.672488\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.65it/s, accuracy=1, cost=2.07]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 403.73it/s, accuracy=0.667, cost=2.69]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.72it/s, accuracy=0.859, cost=2.11]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.102643728256226\n",
      "epoch: 19, training loss: 2.375461, training acc: 0.774823, valid loss: 2.568802, valid acc: 0.696077\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=1.92]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.87it/s, accuracy=0.667, cost=2.55]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.72it/s, accuracy=0.875, cost=1.94]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.1143639087677\n",
      "epoch: 20, training loss: 2.211404, training acc: 0.794137, valid loss: 2.425748, valid acc: 0.712714\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=1.77]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 403.19it/s, accuracy=0.722, cost=2.41]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.62it/s, accuracy=0.922, cost=1.78]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.13349747657776\n",
      "epoch: 21, training loss: 2.054590, training acc: 0.811350, valid loss: 2.289284, valid acc: 0.729843\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=1.63]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.27it/s, accuracy=0.75, cost=2.28] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.65it/s, accuracy=0.922, cost=1.63]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.10236310958862\n",
      "epoch: 22, training loss: 1.905737, training acc: 0.827389, valid loss: 2.159907, valid acc: 0.744513\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=1.5]     \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.95it/s, accuracy=0.75, cost=2.15] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.80it/s, accuracy=0.922, cost=1.49]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11999011039734\n",
      "epoch: 23, training loss: 1.765301, training acc: 0.841571, valid loss: 2.037920, valid acc: 0.756991\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=1.38]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.52it/s, accuracy=0.778, cost=2.04]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.72it/s, accuracy=0.938, cost=1.36]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.10970091819763\n",
      "epoch: 24, training loss: 1.633506, training acc: 0.856018, valid loss: 1.923459, valid acc: 0.768476\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=1.26]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.01it/s, accuracy=0.778, cost=1.93]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.69it/s, accuracy=0.953, cost=1.24]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.130056619644165\n",
      "epoch: 25, training loss: 1.510378, training acc: 0.868208, valid loss: 1.816511, valid acc: 0.778918\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=1.15]     \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.21it/s, accuracy=0.778, cost=1.82]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.62it/s, accuracy=0.953, cost=1.14]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11238241195679\n",
      "epoch: 26, training loss: 1.395778, training acc: 0.878938, valid loss: 1.716944, valid acc: 0.788653\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=1.05]     \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.73it/s, accuracy=0.778, cost=1.73]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.61it/s, accuracy=0.953, cost=1.04]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.10896062850952\n",
      "epoch: 27, training loss: 1.289440, training acc: 0.889248, valid loss: 1.624521, valid acc: 0.796441\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=0.956]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.69it/s, accuracy=0.806, cost=1.64]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.75it/s, accuracy=0.953, cost=0.944]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11905241012573\n",
      "epoch: 28, training loss: 1.191004, training acc: 0.898142, valid loss: 1.538933, valid acc: 0.805359\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=0.868]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.28it/s, accuracy=0.806, cost=1.56]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.65it/s, accuracy=0.969, cost=0.861]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11998915672302\n",
      "epoch: 29, training loss: 1.100053, training acc: 0.906438, valid loss: 1.459815, valid acc: 0.813235\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.786]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.47it/s, accuracy=0.806, cost=1.48]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.68it/s, accuracy=0.984, cost=0.785]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.10346817970276\n",
      "epoch: 30, training loss: 1.016137, training acc: 0.914690, valid loss: 1.386773, valid acc: 0.818810\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.711]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 409.86it/s, accuracy=0.806, cost=1.41]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.72it/s, accuracy=0.984, cost=0.717]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.109402894973755\n",
      "epoch: 31, training loss: 0.938796, training acc: 0.921261, valid loss: 1.319397, valid acc: 0.824120\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.641]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.56it/s, accuracy=0.806, cost=1.35] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.69it/s, accuracy=0.984, cost=0.654]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.112364053726196\n",
      "epoch: 32, training loss: 0.867571, training acc: 0.927566, valid loss: 1.257275, valid acc: 0.829695\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.577]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.80it/s, accuracy=0.833, cost=1.29] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.74it/s, accuracy=0.984, cost=0.598]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11452126502991\n",
      "epoch: 33, training loss: 0.802016, training acc: 0.932832, valid loss: 1.200003, valid acc: 0.834543\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.518]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.35it/s, accuracy=0.833, cost=1.24] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.984, cost=0.547]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.10589838027954\n",
      "epoch: 34, training loss: 0.741706, training acc: 0.937699, valid loss: 1.147191, valid acc: 0.838171\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.465]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.07it/s, accuracy=0.833, cost=1.19] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.60it/s, accuracy=0.984, cost=0.501]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.1006977558136\n",
      "epoch: 35, training loss: 0.686237, training acc: 0.941704, valid loss: 1.098465, valid acc: 0.842065\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.61it/s, accuracy=1, cost=0.416]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.51it/s, accuracy=0.861, cost=1.14] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.62it/s, accuracy=0.984, cost=0.459]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.1454176902771\n",
      "epoch: 36, training loss: 0.635233, training acc: 0.945354, valid loss: 1.053478, valid acc: 0.844877\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.371]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.87it/s, accuracy=0.861, cost=1.1]  \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.76it/s, accuracy=0.984, cost=0.421]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.100993394851685\n",
      "epoch: 37, training loss: 0.588344, training acc: 0.949226, valid loss: 1.011903, valid acc: 0.848771\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.331]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.67it/s, accuracy=0.861, cost=1.06] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.70it/s, accuracy=0.984, cost=0.387]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.110379695892334\n",
      "epoch: 38, training loss: 0.545252, training acc: 0.952323, valid loss: 0.973440, valid acc: 0.851691\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=0.295]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 404.71it/s, accuracy=0.861, cost=1.02] \n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.63it/s, accuracy=0.984, cost=0.355]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11869835853577\n",
      "epoch: 39, training loss: 0.505663, training acc: 0.955398, valid loss: 0.937813, valid acc: 0.854523\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.62it/s, accuracy=1, cost=0.262]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.32it/s, accuracy=0.861, cost=0.985]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.66it/s, accuracy=0.984, cost=0.327]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.134268045425415\n",
      "epoch: 40, training loss: 0.469307, training acc: 0.958407, valid loss: 0.904773, valid acc: 0.857443\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=0.233]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 405.08it/s, accuracy=0.861, cost=0.952]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.74it/s, accuracy=0.984, cost=0.301]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.122865438461304\n",
      "epoch: 41, training loss: 0.435935, training acc: 0.960531, valid loss: 0.874092, valid acc: 0.859302\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.65it/s, accuracy=1, cost=0.207]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.91it/s, accuracy=0.861, cost=0.922]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.77it/s, accuracy=0.984, cost=0.277]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.096822023391724\n",
      "epoch: 42, training loss: 0.405314, training acc: 0.963031, valid loss: 0.845566, valid acc: 0.862222\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.185]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.56it/s, accuracy=0.889, cost=0.894]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.67it/s, accuracy=0.984, cost=0.256]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.1136736869812\n",
      "epoch: 43, training loss: 0.377223, training acc: 0.965575, valid loss: 0.819009, valid acc: 0.865123\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=0.165]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 407.37it/s, accuracy=0.889, cost=0.867]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.66it/s, accuracy=0.984, cost=0.236]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.12402415275574\n",
      "epoch: 44, training loss: 0.351456, training acc: 0.967743, valid loss: 0.794257, valid acc: 0.868043\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.147]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 405.74it/s, accuracy=0.889, cost=0.842]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.76it/s, accuracy=0.984, cost=0.218]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.103622913360596\n",
      "epoch: 45, training loss: 0.327817, training acc: 0.969513, valid loss: 0.771157, valid acc: 0.870079\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.131]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 406.92it/s, accuracy=0.889, cost=0.819]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.54it/s, accuracy=0.984, cost=0.202]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.111342430114746\n",
      "epoch: 46, training loss: 0.306118, training acc: 0.971372, valid loss: 0.749576, valid acc: 0.871937\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.118]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.82it/s, accuracy=0.889, cost=0.796]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.73it/s, accuracy=0.984, cost=0.187]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.111047983169556\n",
      "epoch: 47, training loss: 0.286186, training acc: 0.973208, valid loss: 0.729391, valid acc: 0.874415\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.63it/s, accuracy=1, cost=0.106]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 410.57it/s, accuracy=0.889, cost=0.776]\n",
      "train minibatch loop:   0%|          | 3/707 [00:00<00:32, 21.75it/s, accuracy=0.984, cost=0.174]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.11710834503174\n",
      "epoch: 48, training loss: 0.267859, training acc: 0.974624, valid loss: 0.710492, valid acc: 0.875477\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 707/707 [00:32<00:00, 21.64it/s, accuracy=1, cost=0.095]    \n",
      "test minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 408.73it/s, accuracy=0.889, cost=0.756]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 33.10894179344177\n",
      "epoch: 49, training loss: 0.250987, training acc: 0.976394, valid loss: 0.692780, valid acc: 0.877778\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 64\n",
    "for e in range(50):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, train_X.shape[0], batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        batch_x = convert_sparse_matrix_to_sparse_tensor(train_X[i : min(i + batch_size, train_X.shape[0])])\n",
    "        batch_y = train_Y[i : min(i + batch_size, train_X.shape[0])]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x[0],\n",
    "                model.W: batch_x[1],\n",
    "                model.Y: batch_y\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss += cost\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "\n",
    "    pbar = tqdm(range(0, test_X.shape[0], batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x = convert_sparse_matrix_to_sparse_tensor(test_X[i : min(i + batch_size, test_X.shape[0])])\n",
    "        batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])]\n",
    "        batch_x_expand = np.expand_dims(batch_x,axis = 1)\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x[0],\n",
    "                model.W: batch_x[1],\n",
    "                model.Y: batch_y\n",
    "            },\n",
    "        )\n",
    "        test_loss += cost\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "\n",
    "    train_loss /= train_X.shape[0] / batch_size\n",
    "    train_acc /= train_X.shape[0] / batch_size\n",
    "    test_loss /= test_X.shape[0] / batch_size\n",
    "    test_acc /= test_X.shape[0] / batch_size\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (e, train_loss, train_acc, test_loss, test_acc)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 177/177 [00:00<00:00, 759.90it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, test_X.shape[0], batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    batch_x = convert_sparse_matrix_to_sparse_tensor(test_X[i : min(i + batch_size, test_X.shape[0])])\n",
    "    batch_y = test_Y[i : min(i + batch_size, test_X.shape[0])].tolist()\n",
    "    predict_Y += np.argmax(\n",
    "        sess.run(\n",
    "            model.logits, feed_dict = {model.X: batch_x[0], model.W: batch_x[1], model.Y: batch_y}\n",
    "        ),\n",
    "        1,\n",
    "    ).tolist()\n",
    "    real_Y += batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         afr       0.92      0.92      0.92        88\n",
      "         ara       0.86      0.84      0.85       114\n",
      "         arq       0.88      0.88      0.88       105\n",
      "         arz       0.72      0.74      0.73        98\n",
      "         avk       0.93      0.87      0.90       105\n",
      "         aze       0.92      0.92      0.92       104\n",
      "         bel       0.91      0.89      0.90       111\n",
      "         ben       1.00      1.00      1.00       108\n",
      "         ber       0.88      0.86      0.87       103\n",
      "         bre       0.94      0.96      0.95       102\n",
      "         bul       0.79      0.88      0.83       113\n",
      "         cat       0.79      0.82      0.80       116\n",
      "         cbk       0.99      0.92      0.95       115\n",
      "         ceb       0.83      0.80      0.81       108\n",
      "         ces       0.79      0.75      0.77       111\n",
      "         chv       0.97      0.94      0.96       109\n",
      "         cmn       0.47      0.19      0.27        99\n",
      "         cor       0.93      0.90      0.92       102\n",
      "         csb       0.94      0.93      0.94        88\n",
      "         cym       0.99      0.97      0.98       107\n",
      "         dan       0.73      0.82      0.77        90\n",
      "         deu       0.96      0.97      0.96        93\n",
      "         dsb       0.82      0.87      0.84       105\n",
      "         dtp       0.89      0.82      0.85       100\n",
      "         ell       0.97      0.95      0.96        99\n",
      "         eng       0.93      0.95      0.94       104\n",
      "         epo       0.86      0.91      0.88       106\n",
      "         est       0.87      0.97      0.92        92\n",
      "         eus       0.91      0.96      0.93       102\n",
      "         fin       0.90      0.98      0.94       105\n",
      "         fra       0.92      0.96      0.94        95\n",
      "         gla       0.89      0.96      0.93        84\n",
      "         gle       0.97      0.94      0.95        98\n",
      "         glg       0.75      0.80      0.77       110\n",
      "         grc       0.94      0.97      0.95        91\n",
      "         grn       0.91      0.94      0.93       101\n",
      "         heb       0.98      0.98      0.98        96\n",
      "         hin       0.97      1.00      0.98        94\n",
      "         hoc       0.98      0.94      0.96        99\n",
      "         hrv       0.52      0.59      0.56        91\n",
      "         hsb       0.77      0.80      0.79       107\n",
      "         hun       0.92      0.92      0.92       101\n",
      "         hye       0.99      0.99      0.99        80\n",
      "         ido       0.86      0.90      0.88       110\n",
      "         ile       0.84      0.84      0.84        89\n",
      "         ilo       0.89      0.95      0.92       103\n",
      "         ina       0.86      0.90      0.88        92\n",
      "         ind       0.79      0.61      0.69       109\n",
      "         isl       0.97      0.97      0.97        97\n",
      "         ita       0.81      0.88      0.84       104\n",
      "         jbo       0.95      0.93      0.94       107\n",
      "         jpn       0.81      0.73      0.77        93\n",
      "         kab       0.80      0.84      0.82        93\n",
      "         kat       0.97      1.00      0.99       104\n",
      "         kaz       0.97      0.99      0.98       105\n",
      "         kha       0.98      0.92      0.95       109\n",
      "         khm       0.94      0.96      0.95        94\n",
      "         kor       0.81      0.75      0.78       102\n",
      "         kur       0.82      0.92      0.87        91\n",
      "         kzj       0.85      0.94      0.89       105\n",
      "         lat       0.79      0.81      0.80        97\n",
      "         lfn       0.82      0.92      0.86       107\n",
      "         lit       0.86      0.96      0.91       108\n",
      "         lvs       0.90      0.93      0.92       101\n",
      "         lzh       0.64      0.23      0.34       101\n",
      "         mal       0.99      1.00      0.99        95\n",
      "         mar       0.99      0.99      0.99       102\n",
      "         mhr       0.97      0.98      0.98       107\n",
      "         mkd       0.78      0.84      0.81       106\n",
      "         mon       1.00      0.99      1.00       109\n",
      "         nds       0.88      0.92      0.90        95\n",
      "         nld       0.88      0.94      0.91        93\n",
      "         nno       0.75      0.84      0.80        95\n",
      "         nob       0.72      0.57      0.64       105\n",
      "         nst       1.00      1.00      1.00        95\n",
      "         oci       0.79      0.84      0.81        85\n",
      "         orv       0.95      0.98      0.97       100\n",
      "         pam       0.93      0.94      0.93        99\n",
      "         pes       0.96      0.97      0.96        88\n",
      "         pms       0.83      0.91      0.87       106\n",
      "         pol       0.93      0.93      0.93       105\n",
      "         por       0.78      0.85      0.81        89\n",
      "         prg       0.96      0.99      0.97       108\n",
      "         rom       0.94      0.83      0.88       105\n",
      "         ron       0.92      0.91      0.91       118\n",
      "         rus       0.84      0.87      0.86        95\n",
      "         sah       0.96      0.98      0.97        84\n",
      "         slk       0.78      0.80      0.79       100\n",
      "         slv       0.59      0.75      0.66       102\n",
      "         spa       0.86      0.83      0.84       110\n",
      "         sqi       0.93      0.93      0.93        99\n",
      "         srp       0.51      0.33      0.40        99\n",
      "         swe       0.88      0.90      0.89       109\n",
      "         tat       0.95      0.93      0.94       105\n",
      "         tgl       0.79      0.94      0.86        90\n",
      "         tha       0.94      0.98      0.96       101\n",
      "         tlh       0.87      0.88      0.87        81\n",
      "        toki       1.00      0.98      0.99        96\n",
      "         tpw       0.93      0.94      0.94        90\n",
      "         tur       0.85      0.91      0.88        94\n",
      "         uig       0.97      0.98      0.97        98\n",
      "         ukr       0.83      0.91      0.87        86\n",
      "         urd       1.00      0.97      0.99       103\n",
      "         uzb       0.82      0.88      0.85        91\n",
      "         vie       0.98      0.98      0.98        99\n",
      "         vol       0.93      0.94      0.93       110\n",
      "         war       0.87      0.87      0.87        92\n",
      "         wuu       0.52      0.32      0.39        92\n",
      "         xal       0.97      0.99      0.98       100\n",
      "         yid       0.97      0.99      0.98       108\n",
      "         yue       0.51      0.31      0.39        89\n",
      "         zsm       0.66      0.82      0.73       103\n",
      "         zza       0.94      0.91      0.92        99\n",
      "\n",
      "   micro avg       0.88      0.88      0.88     11300\n",
      "   macro avg       0.87      0.88      0.87     11300\n",
      "weighted avg       0.87      0.88      0.87     11300\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, predict_Y, target_names = labels\n",
    "    )\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
